{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. 读入数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "调用系统提供的Mnist数据函数读入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-7aa510212b84>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\Users\\H\\Install\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\Users\\H\\Install\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting C:/Users/H/Install/Anaconda3/Lib/site-packages/tensorflow/examples/tutorials/mnist\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\H\\Install\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting C:/Users/H/Install/Anaconda3/Lib/site-packages/tensorflow/examples/tutorials/mnist\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\H\\Install\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting C:/Users/H/Install/Anaconda3/Lib/site-packages/tensorflow/examples/tutorials/mnist\\t10k-images-idx3-ubyte.gz\n",
      "Extracting C:/Users/H/Install/Anaconda3/Lib/site-packages/tensorflow/examples/tutorials/mnist\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\H\\Install\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = 'C:/Users/H/Install/Anaconda3/Lib/site-packages/tensorflow/examples/tutorials/mnist'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 构建模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "确定隐层数量和每层的神经元数量并初始化权重；确定激活函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create the model\n",
    "# 输入层\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "# 隐层1（200个神经元）\n",
    "W_1 = tf.Variable(tf.random_normal([784, 200], stddev=0.1, seed=1))\n",
    "b_1 = tf.Variable(tf.random_normal([200], stddev=0.1, seed=1))\n",
    "h_1 = tf.tanh(tf.matmul(x, W_1) + b_1)\n",
    "\n",
    "# 隐层2（100个神经元）\n",
    "W_2 = tf.Variable(tf.random_normal([200, 100], stddev=0.1, seed=2))\n",
    "b_2 = tf.Variable(tf.random_normal([100], stddev=0.1, seed=2))\n",
    "h_2 = tf.tanh(tf.matmul(h_1, W_2) + b_2)\n",
    "\n",
    "# 输出层\n",
    "W_3 = tf.Variable(tf.random_normal([100, 10], stddev=0.1, seed=3))\n",
    "b_3 = tf.Variable(tf.random_normal([10], stddev=0.1, seed=3))\n",
    "y = tf.matmul(h_2, W_3) + b_3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义ground_truth占位符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义损失函数\n",
    "\n",
    "注意交叉熵不要使用注释中的手动计算方式，而是使用系统函数。\n",
    "另一点就是，softmax_cross_entropy_with_logits的logits参数是未经激活的wx+b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "\n",
    "# 设置正则化参数\n",
    "Lambda2 = tf.constant(0.00002)\n",
    "# Lambda1 = tf.constant(0.000000001)\n",
    "\n",
    "# 计算损失函数（交叉熵+正则项）\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=y) + Lambda2/2 * (\n",
    "                                    tf.reduce_sum(tf.square(W_1)) + tf.reduce_sum(tf.square(W_2)) + tf.reduce_sum(tf.square(W_3))))\n",
    "\n",
    "# L2正则：Lambda2 * (tf.reduce_sum(tf.square(W_1)) + tf.reduce_sum(tf.square(W_2)) + tf.reduce_sum(tf.square(W_3))))\n",
    "# L1正则：ambda1 * (tf.reduce_sum(tf.abs(W_1)) + tf.reduce_sum(tf.abs(W_2)) + tf.reduce_sum(tf.abs(W_3))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成一个训练step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 学习率逐渐降低（每个epoch更新一次）\n",
    "epoch = tf.placeholder(tf.float32)\n",
    "train_step = tf.train.GradientDescentOptimizer(0.7/tf.sqrt(epoch+1)).minimize(cross_entropy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义模型性能评价函数（此处为预测准确率）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "调用系统提供的读取数据，取得一个batch。 然后运行6k个step(10 epochs)，对权重进行优化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3191\n",
      "0.9494\n",
      "0.9586\n",
      "0.9673\n",
      "0.9716\n",
      "0.9727\n",
      "0.9766\n",
      "0.9754\n",
      "0.9777\n",
      "0.9778\n",
      "0.9774\n",
      "0.9773\n",
      "0.9787\n",
      "0.979\n",
      "0.9792\n",
      "0.9806\n",
      "0.9799\n",
      "0.98\n",
      "0.981\n",
      "0.9814\n"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "for _ in range(6000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys, epoch: _//600})\n",
    "    if _%300 == 0:\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels}))     # 每运行300次输出一次测试准确率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 模型测试"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "验证模型在测试数据上的准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9806\n"
     ]
    }
   ],
   "source": [
    "# Test trained model\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "参数汇总：\n",
    "\n",
    "- 隐层神经元数量：共设置两个隐层，第一个隐层由200个神经元构成，第二个隐层由100个神经元构成\n",
    "- 权重初始化：所有权重初始化值服从均值为0，标准差为0.1的高斯分布\n",
    "- 激活函数：隐层采用双曲正切函数tanh，输出层采用softmax\n",
    "- 正则化：采用L2正则，正则化参数为0.00002\n",
    "- 学习率：采用学习率逐渐降低的策略，初始值较大，设为0.7，每一个epoch更新一次，更新规则为 𝜂𝑡 = 𝜂/sqrt(epoch+1)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
